{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "!pip install langchain langchain-core pandas numpy matplotlib seaborn scikit-learn\n",
        "\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.cluster import KMeans\n",
        "from sklearn.metrics import silhouette_score\n",
        "from typing import Dict, List, Tuple, Optional, Any\n",
        "from langchain_core.tools import BaseTool\n",
        "from langchain_core.tools.base import ToolException\n",
        "from pydantic import BaseModel, Field\n",
        "import json"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YttkxDuyFo0y",
        "outputId": "91072452-13d4-4a78-e2c5-e465db916cce"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: langchain in /usr/local/lib/python3.11/dist-packages (0.3.25)\n",
            "Requirement already satisfied: langchain-core in /usr/local/lib/python3.11/dist-packages (0.3.63)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.11/dist-packages (2.2.2)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (2.0.2)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.11/dist-packages (3.10.0)\n",
            "Requirement already satisfied: seaborn in /usr/local/lib/python3.11/dist-packages (0.13.2)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.11/dist-packages (1.6.1)\n",
            "Requirement already satisfied: langchain-text-splitters<1.0.0,>=0.3.8 in /usr/local/lib/python3.11/dist-packages (from langchain) (0.3.8)\n",
            "Requirement already satisfied: langsmith<0.4,>=0.1.17 in /usr/local/lib/python3.11/dist-packages (from langchain) (0.3.44)\n",
            "Requirement already satisfied: pydantic<3.0.0,>=2.7.4 in /usr/local/lib/python3.11/dist-packages (from langchain) (2.11.5)\n",
            "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /usr/local/lib/python3.11/dist-packages (from langchain) (2.0.41)\n",
            "Requirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.11/dist-packages (from langchain) (2.32.3)\n",
            "Requirement already satisfied: PyYAML>=5.3 in /usr/local/lib/python3.11/dist-packages (from langchain) (6.0.2)\n",
            "Requirement already satisfied: tenacity!=8.4.0,<10.0.0,>=8.1.0 in /usr/local/lib/python3.11/dist-packages (from langchain-core) (9.1.2)\n",
            "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /usr/local/lib/python3.11/dist-packages (from langchain-core) (1.33)\n",
            "Requirement already satisfied: packaging<25,>=23.2 in /usr/local/lib/python3.11/dist-packages (from langchain-core) (24.2)\n",
            "Requirement already satisfied: typing-extensions>=4.7 in /usr/local/lib/python3.11/dist-packages (from langchain-core) (4.14.0)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas) (2.9.0.post0)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas) (2025.2)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas) (2025.2)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.3.2)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (4.58.1)\n",
            "Requirement already satisfied: kiwisolver>=1.3.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.4.8)\n",
            "Requirement already satisfied: pillow>=8 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (11.2.1)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (3.2.3)\n",
            "Requirement already satisfied: scipy>=1.6.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (1.15.3)\n",
            "Requirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (1.5.1)\n",
            "Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (3.6.0)\n",
            "Requirement already satisfied: jsonpointer>=1.9 in /usr/local/lib/python3.11/dist-packages (from jsonpatch<2.0,>=1.33->langchain-core) (3.0.0)\n",
            "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.11/dist-packages (from langsmith<0.4,>=0.1.17->langchain) (0.28.1)\n",
            "Requirement already satisfied: orjson<4.0.0,>=3.9.14 in /usr/local/lib/python3.11/dist-packages (from langsmith<0.4,>=0.1.17->langchain) (3.10.18)\n",
            "Requirement already satisfied: requests-toolbelt<2.0.0,>=1.0.0 in /usr/local/lib/python3.11/dist-packages (from langsmith<0.4,>=0.1.17->langchain) (1.0.0)\n",
            "Requirement already satisfied: zstandard<0.24.0,>=0.23.0 in /usr/local/lib/python3.11/dist-packages (from langsmith<0.4,>=0.1.17->langchain) (0.23.0)\n",
            "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.11/dist-packages (from pydantic<3.0.0,>=2.7.4->langchain) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.33.2 in /usr/local/lib/python3.11/dist-packages (from pydantic<3.0.0,>=2.7.4->langchain) (2.33.2)\n",
            "Requirement already satisfied: typing-inspection>=0.4.0 in /usr/local/lib/python3.11/dist-packages (from pydantic<3.0.0,>=2.7.4->langchain) (0.4.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.11/dist-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2->langchain) (3.4.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2->langchain) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2->langchain) (2.4.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2->langchain) (2025.4.26)\n",
            "Requirement already satisfied: greenlet>=1 in /usr/local/lib/python3.11/dist-packages (from SQLAlchemy<3,>=1.4->langchain) (3.2.2)\n",
            "Requirement already satisfied: anyio in /usr/local/lib/python3.11/dist-packages (from httpx<1,>=0.23.0->langsmith<0.4,>=0.1.17->langchain) (4.9.0)\n",
            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.11/dist-packages (from httpx<1,>=0.23.0->langsmith<0.4,>=0.1.17->langchain) (1.0.9)\n",
            "Requirement already satisfied: h11>=0.16 in /usr/local/lib/python3.11/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->langsmith<0.4,>=0.1.17->langchain) (0.16.0)\n",
            "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.11/dist-packages (from anyio->httpx<1,>=0.23.0->langsmith<0.4,>=0.1.17->langchain) (1.3.1)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class DataAnalysisInput(BaseModel):\n",
        "    data: List[Dict[str, Any]] = Field(description=\"List of data records as dictionaries\")\n",
        "    analysis_type: str = Field(default=\"comprehensive\", description=\"Type of analysis: 'comprehensive', 'clustering', 'correlation', 'outlier'\")\n",
        "    target_column: Optional[str] = Field(default=None, description=\"Target column for focused analysis\")\n",
        "    max_clusters: int = Field(default=5, description=\"Maximum clusters for clustering analysis\")"
      ],
      "metadata": {
        "id": "5CkxvLIMFpoP"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class IntelligentDataAnalyzer(BaseTool):\n",
        "    name: str = \"intelligent_data_analyzer\"\n",
        "    description: str = \"Advanced data analysis tool that performs statistical analysis, machine learning clustering, outlier detection, correlation analysis, and generates visualizations with actionable insights.\"\n",
        "    args_schema: type[BaseModel] = DataAnalysisInput\n",
        "    response_format: str = \"content_and_artifact\"\n",
        "\n",
        "    def _run(self, data: List[Dict], analysis_type: str = \"comprehensive\", target_column: Optional[str] = None, max_clusters: int = 5) -> Tuple[str, Dict]:\n",
        "        try:\n",
        "            df = pd.DataFrame(data)\n",
        "            if df.empty:\n",
        "                raise ToolException(\"Dataset is empty\")\n",
        "\n",
        "            insights = {\"dataset_info\": self._get_dataset_info(df)}\n",
        "\n",
        "            if analysis_type in [\"comprehensive\", \"correlation\"]:\n",
        "                insights[\"correlation_analysis\"] = self._correlation_analysis(df)\n",
        "            if analysis_type in [\"comprehensive\", \"clustering\"]:\n",
        "                insights[\"clustering_analysis\"] = self._clustering_analysis(df, max_clusters)\n",
        "            if analysis_type in [\"comprehensive\", \"outlier\"]:\n",
        "                insights[\"outlier_detection\"] = self._outlier_detection(df)\n",
        "\n",
        "            if target_column and target_column in df.columns:\n",
        "                insights[\"target_analysis\"] = self._target_analysis(df, target_column)\n",
        "\n",
        "            recommendations = self._generate_recommendations(df, insights)\n",
        "            summary = self._create_analysis_summary(insights, recommendations)\n",
        "\n",
        "            artifact = {\n",
        "                \"insights\": insights,\n",
        "                \"recommendations\": recommendations,\n",
        "                \"data_shape\": df.shape,\n",
        "                \"analysis_type\": analysis_type,\n",
        "                \"numeric_columns\": df.select_dtypes(include=[np.number]).columns.tolist(),\n",
        "                \"categorical_columns\": df.select_dtypes(include=['object']).columns.tolist()\n",
        "            }\n",
        "\n",
        "            return summary, artifact\n",
        "\n",
        "        except Exception as e:\n",
        "            raise ToolException(f\"Analysis failed: {str(e)}\")\n",
        "\n",
        "    def _get_dataset_info(self, df: pd.DataFrame) -> Dict:\n",
        "        return {\n",
        "            \"shape\": df.shape,\n",
        "            \"columns\": df.columns.tolist(),\n",
        "            \"dtypes\": df.dtypes.astype(str).to_dict(),\n",
        "            \"missing_values\": df.isnull().sum().to_dict(),\n",
        "            \"memory_usage\": df.memory_usage(deep=True).sum()\n",
        "        }\n",
        "\n",
        "    def _correlation_analysis(self, df: pd.DataFrame) -> Dict:\n",
        "        numeric_df = df.select_dtypes(include=[np.number])\n",
        "        if numeric_df.empty:\n",
        "            return {\"message\": \"No numeric columns for correlation analysis\"}\n",
        "\n",
        "        corr_matrix = numeric_df.corr()\n",
        "        strong_corr = []\n",
        "        for i in range(len(corr_matrix.columns)):\n",
        "            for j in range(i+1, len(corr_matrix.columns)):\n",
        "                corr_val = corr_matrix.iloc[i, j]\n",
        "                if abs(corr_val) > 0.7:\n",
        "                    strong_corr.append({\"var1\": corr_matrix.columns[i], \"var2\": corr_matrix.columns[j], \"correlation\": round(corr_val, 3)})\n",
        "\n",
        "        return {\n",
        "            \"correlation_matrix\": corr_matrix.round(3).to_dict(),\n",
        "            \"strong_correlations\": strong_corr,\n",
        "            \"avg_correlation\": round(corr_matrix.values[np.triu_indices_from(corr_matrix.values, k=1)].mean(), 3)\n",
        "        }\n",
        "\n",
        "    def _clustering_analysis(self, df: pd.DataFrame, max_clusters: int) -> Dict:\n",
        "        numeric_df = df.select_dtypes(include=[np.number]).dropna()\n",
        "        if numeric_df.shape[0] < 2 or numeric_df.shape[1] < 2:\n",
        "            return {\"message\": \"Insufficient numeric data for clustering\"}\n",
        "\n",
        "        scaler = StandardScaler()\n",
        "        scaled_data = scaler.fit_transform(numeric_df)\n",
        "\n",
        "        inertias = []\n",
        "        K_range = range(1, min(max_clusters + 1, len(numeric_df) // 2 + 1))\n",
        "\n",
        "        for k in K_range:\n",
        "            kmeans = KMeans(n_clusters=k, random_state=42, n_init=10)\n",
        "            kmeans.fit(scaled_data)\n",
        "            inertias.append(kmeans.inertia_)\n",
        "\n",
        "        optimal_k = self._find_elbow_point(inertias, K_range)\n",
        "        kmeans = KMeans(n_clusters=optimal_k, random_state=42, n_init=10)\n",
        "        cluster_labels = kmeans.fit_predict(scaled_data)\n",
        "\n",
        "        cluster_stats = {}\n",
        "        for i in range(optimal_k):\n",
        "            cluster_data = numeric_df[cluster_labels == i]\n",
        "            cluster_stats[f\"cluster_{i}\"] = {\n",
        "                \"size\": len(cluster_data),\n",
        "                \"percentage\": round(len(cluster_data) / len(numeric_df) * 100, 1),\n",
        "                \"means\": cluster_data.mean().round(3).to_dict()\n",
        "            }\n",
        "\n",
        "        return {\n",
        "            \"optimal_clusters\": optimal_k,\n",
        "            \"cluster_stats\": cluster_stats,\n",
        "            \"silhouette_score\": round(silhouette_score(scaled_data, cluster_labels), 3) if len(set(cluster_labels)) > 1 else 0.0,\n",
        "            \"inertias\": inertias\n",
        "        }\n",
        "\n",
        "    def _outlier_detection(self, df: pd.DataFrame) -> Dict:\n",
        "        numeric_df = df.select_dtypes(include=[np.number])\n",
        "        if numeric_df.empty:\n",
        "            return {\"message\": \"No numeric columns for outlier detection\"}\n",
        "\n",
        "        outliers = {}\n",
        "        for col in numeric_df.columns:\n",
        "            data = numeric_df[col].dropna()\n",
        "            Q1, Q3 = data.quantile(0.25), data.quantile(0.75)\n",
        "            IQR = Q3 - Q1\n",
        "            iqr_outliers = data[(data < Q1 - 1.5 * IQR) | (data > Q3 + 1.5 * IQR)]\n",
        "            z_scores = np.abs((data - data.mean()) / data.std())\n",
        "            z_outliers = data[z_scores > 3]\n",
        "\n",
        "            outliers[col] = {\n",
        "                \"iqr_outliers\": len(iqr_outliers),\n",
        "                \"z_score_outliers\": len(z_outliers),\n",
        "                \"outlier_percentage\": round(len(iqr_outliers) / len(data) * 100, 2)\n",
        "            }\n",
        "\n",
        "        return outliers\n",
        "\n",
        "    def _target_analysis(self, df: pd.DataFrame, target_col: str) -> Dict:\n",
        "        if target_col not in df.columns:\n",
        "            return {\"error\": f\"Column {target_col} not found\"}\n",
        "\n",
        "        target_data = df[target_col].dropna()\n",
        "\n",
        "        if pd.api.types.is_numeric_dtype(target_data):\n",
        "            return {\n",
        "                \"type\": \"numeric\",\n",
        "                \"stats\": {\n",
        "                    \"mean\": round(target_data.mean(), 3),\n",
        "                    \"median\": round(target_data.median(), 3),\n",
        "                    \"std\": round(target_data.std(), 3),\n",
        "                    \"skewness\": round(target_data.skew(), 3),\n",
        "                    \"kurtosis\": round(target_data.kurtosis(), 3)\n",
        "                },\n",
        "                \"distribution\": \"normal\" if abs(target_data.skew()) < 0.5 else \"skewed\"\n",
        "            }\n",
        "        else:\n",
        "            value_counts = target_data.value_counts()\n",
        "            return {\n",
        "                \"type\": \"categorical\",\n",
        "                \"unique_values\": len(value_counts),\n",
        "                \"most_common\": value_counts.head(5).to_dict(),\n",
        "                \"entropy\": round(-sum((p := value_counts / len(target_data)) * np.log2(p + 1e-10)), 3)\n",
        "            }\n",
        "\n",
        "    def _generate_recommendations(self, df: pd.DataFrame, insights: Dict) -> List[str]:\n",
        "        recommendations = []\n",
        "\n",
        "        missing_pct = sum(insights[\"dataset_info\"][\"missing_values\"].values()) / (df.shape[0] * df.shape[1]) * 100\n",
        "        if missing_pct > 10:\n",
        "            recommendations.append(f\"Consider data imputation - {missing_pct:.1f}% missing values detected\")\n",
        "\n",
        "        if \"correlation_analysis\" in insights and insights[\"correlation_analysis\"].get(\"strong_correlations\"):\n",
        "            recommendations.append(\"Strong correlations detected - consider feature selection or dimensionality reduction\")\n",
        "\n",
        "        if \"clustering_analysis\" in insights:\n",
        "            cluster_info = insights[\"clustering_analysis\"]\n",
        "            if isinstance(cluster_info, dict) and \"optimal_clusters\" in cluster_info:\n",
        "                recommendations.append(f\"Data segments into {cluster_info['optimal_clusters']} distinct groups - useful for targeted strategies\")\n",
        "\n",
        "        if \"outlier_detection\" in insights:\n",
        "            high_outlier_cols = [col for col, info in insights[\"outlier_detection\"].items() if isinstance(info, dict) and info.get(\"outlier_percentage\", 0) > 5]\n",
        "            if high_outlier_cols:\n",
        "                recommendations.append(f\"High outlier percentage in: {', '.join(high_outlier_cols)} - investigate data quality\")\n",
        "\n",
        "        return recommendations if recommendations else [\"Data appears well-structured with no immediate concerns\"]\n",
        "\n",
        "    def _create_analysis_summary(self, insights: Dict, recommendations: List[str]) -> str:\n",
        "        dataset_info = insights[\"dataset_info\"]\n",
        "        summary = f\"\"\"📊 INTELLIGENT DATA ANALYSIS COMPLETE\n",
        "\n",
        "Dataset Overview: {dataset_info['shape'][0]} rows × {dataset_info['shape'][1]} columns\n",
        "Numeric Features: {len([c for c, t in dataset_info['dtypes'].items() if 'int' in t or 'float' in t])}\n",
        "Categorical Features: {len([c for c, t in dataset_info['dtypes'].items() if 'object' in t])}\n",
        "\n",
        "Key Insights Generated:\n",
        "• Statistical correlations and relationships identified\n",
        "• Clustering patterns discovered for segmentation\n",
        "• Outlier detection completed for data quality assessment\n",
        "• Feature importance and distribution analysis performed\n",
        "\n",
        "Top Recommendations:\n",
        "{chr(10).join('• ' + rec for rec in recommendations[:3])}\n",
        "\n",
        "Analysis includes ML-powered clustering, statistical correlations, and actionable business insights.\"\"\"\n",
        "\n",
        "        return summary\n",
        "\n",
        "    def _find_elbow_point(self, inertias: List[float], k_range: range) -> int:\n",
        "        if len(inertias) < 3:\n",
        "            return list(k_range)[0]\n",
        "        diffs = [inertias[i-1] - inertias[i] for i in range(1, len(inertias))]\n",
        "        return list(k_range)[diffs.index(max(diffs)) + 1] if diffs else list(k_range)[0]"
      ],
      "metadata": {
        "id": "7qvlrl7tFry_"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8mSPvLRb4w2k",
        "outputId": "630239b1-b85b-41e5-86cb-d89dfc695d08"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Analysis Summary:\n",
            "📊 INTELLIGENT DATA ANALYSIS COMPLETE\n",
            "\n",
            "Dataset Overview: 5 rows × 4 columns\n",
            "Numeric Features: 3\n",
            "Categorical Features: 1\n",
            "\n",
            "Key Insights Generated:\n",
            "• Statistical correlations and relationships identified\n",
            "• Clustering patterns discovered for segmentation\n",
            "• Outlier detection completed for data quality assessment\n",
            "• Feature importance and distribution analysis performed\n",
            "\n",
            "Top Recommendations:\n",
            "• Strong correlations detected - consider feature selection or dimensionality reduction\n",
            "• Data segments into 1 distinct groups - useful for targeted strategies\n",
            "\n",
            "Analysis includes ML-powered clustering, statistical correlations, and actionable business insights.\n"
          ]
        }
      ],
      "source": [
        "data_analyzer = IntelligentDataAnalyzer()\n",
        "\n",
        "sample_data = [\n",
        "    {\"age\": 25, \"income\": 50000, \"education\": \"Bachelor\", \"satisfaction\": 7},\n",
        "    {\"age\": 35, \"income\": 75000, \"education\": \"Master\", \"satisfaction\": 8},\n",
        "    {\"age\": 45, \"income\": 90000, \"education\": \"PhD\", \"satisfaction\": 6},\n",
        "    {\"age\": 28, \"income\": 45000, \"education\": \"Bachelor\", \"satisfaction\": 7},\n",
        "    {\"age\": 52, \"income\": 120000, \"education\": \"Master\", \"satisfaction\": 9},\n",
        "]\n",
        "\n",
        "result = data_analyzer.invoke({\n",
        "    \"data\": sample_data,\n",
        "    \"analysis_type\": \"comprehensive\",\n",
        "    \"target_column\": \"satisfaction\"\n",
        "})\n",
        "\n",
        "print(\"Analysis Summary:\")\n",
        "print(result)"
      ]
    }
  ]
}